{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 糖尿病分类——Logistic 回归¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants_tfidf</th>\n",
       "      <th>Plasma_glucose_concentration_tfidf</th>\n",
       "      <th>blood_pressure_tfidf</th>\n",
       "      <th>Triceps_skin_fold_thickness_tfidf</th>\n",
       "      <th>serum_insulin_tfidf</th>\n",
       "      <th>BMI_tfidf</th>\n",
       "      <th>Diabetes_pedigree_function_tfidf</th>\n",
       "      <th>Age_tfidf</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants_tfidf  Plasma_glucose_concentration_tfidf  blood_pressure_tfidf  \\\n",
       "0         0.639947                            0.866045             -0.031990   \n",
       "1        -0.844885                           -1.205066             -0.528319   \n",
       "2         1.233880                            2.016662             -0.693761   \n",
       "3        -0.844885                           -1.073567             -0.528319   \n",
       "4        -1.141852                            0.504422             -2.679076   \n",
       "\n",
       "   Triceps_skin_fold_thickness_tfidf  serum_insulin_tfidf  BMI_tfidf  \\\n",
       "0                           0.670643            -0.181541   0.166619   \n",
       "1                          -0.012301            -0.181541  -0.852200   \n",
       "2                          -0.012301            -0.181541  -1.332500   \n",
       "3                          -0.695245            -0.540642  -0.633881   \n",
       "4                           0.670643             0.316566   1.549303   \n",
       "\n",
       "   Diabetes_pedigree_function_tfidf  Age_tfidf  Target  \n",
       "0                          0.468492   1.425995       1  \n",
       "1                         -0.365061  -0.190672       0  \n",
       "2                          0.604397  -0.105584       1  \n",
       "3                         -0.920763  -1.041549       0  \n",
       "4                          5.484909  -0.020496       1  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# 请自行在log(x+1)特征和tf_idf特征上尝试，并比较不同特征的结果，\n",
    "# path to where the data lies\n",
    "train = pd.read_csv(\"FE_pima-indians-diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']   \n",
    "X_train = train.drop([ \"Target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "#查看一个学习器是否支持稀疏数据，可以看fit函数是否支持: X: {array-like, sparse matrix}.\n",
    "#可自行用timeit比较稠密数据和稀疏数据的训练时间\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.默认参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.48797856 0.53011593 0.4562292  0.422546   0.48392885]\n",
      "cv logloss is: 0.47615970944434044\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()\n",
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "#数据集比较大，采用5折交叉验证\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 正则化的Logistic Regression + GridSearchCV及参数调优\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） \n",
    "目标函数为：J =  C* sum(logloss(f(xi), yi)) + penalty \n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同：\n",
    "1. 设置参数搜索范围\n",
    "2. 生成学习器实例（参数设置）\n",
    "3. 生成GridSearchCV的实例（参数设置）\n",
    "4. 调用GridSearchCV的fit方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 14 candidates, totalling 70 fits\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................ C=0.001, penalty=l1, score=-0.693, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................ C=0.001, penalty=l1, score=-0.693, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................ C=0.001, penalty=l1, score=-0.693, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................ C=0.001, penalty=l1, score=-0.693, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................ C=0.001, penalty=l1, score=-0.693, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................ C=0.001, penalty=l2, score=-0.625, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................ C=0.001, penalty=l2, score=-0.634, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................ C=0.001, penalty=l2, score=-0.626, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................ C=0.001, penalty=l2, score=-0.621, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................ C=0.001, penalty=l2, score=-0.633, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l1, score=-0.636, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l1, score=-0.645, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l1, score=-0.633, total=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l1, score=-0.630, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l1, score=-0.636, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l2, score=-0.512, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l2, score=-0.543, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l2, score=-0.507, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l2, score=-0.487, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l2, score=-0.525, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l1, score=-0.489, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l1, score=-0.525, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l1, score=-0.458, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l1, score=-0.433, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l1, score=-0.487, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l2, score=-0.484, total=   0.0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l2, score=-0.525, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l2, score=-0.460, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l2, score=-0.429, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l2, score=-0.485, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] .................... C=1, penalty=l1, score=-0.488, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] .................... C=1, penalty=l1, score=-0.529, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] .................... C=1, penalty=l1, score=-0.456, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] .................... C=1, penalty=l1, score=-0.422, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] .................... C=1, penalty=l1, score=-0.485, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] .................... C=1, penalty=l2, score=-0.488, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] .................... C=1, penalty=l2, score=-0.530, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] .................... C=1, penalty=l2, score=-0.456, total=   0.0s\n",
      "[CV] C=1, penalty=l2 ................................................."
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[CV] .................... C=1, penalty=l2, score=-0.423, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] .................... C=1, penalty=l2, score=-0.484, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ................... C=10, penalty=l1, score=-0.489, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ................... C=10, penalty=l1, score=-0.531, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ................... C=10, penalty=l1, score=-0.456, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ................... C=10, penalty=l1, score=-0.422, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ................... C=10, penalty=l1, score=-0.484, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ................... C=10, penalty=l2, score=-0.489, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ................... C=10, penalty=l2, score=-0.531, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ................... C=10, penalty=l2, score=-0.456, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ................... C=10, penalty=l2, score=-0.422, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ................... C=10, penalty=l2, score=-0.484, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] .................. C=100, penalty=l1, score=-0.489, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .................. C=100, penalty=l1, score=-0.531, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] .................. C=100, penalty=l1, score=-0.456, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] .................. C=100, penalty=l1, score=-0.422, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] .................. C=100, penalty=l1, score=-0.484, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .................. C=100, penalty=l2, score=-0.489, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .................. C=100, penalty=l2, score=-0.531, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .................. C=100, penalty=l2, score=-0.456, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .................. C=100, penalty=l2, score=-0.422, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .................. C=100, penalty=l2, score=-0.484, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ................. C=1000, penalty=l1, score=-0.489, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ................. C=1000, penalty=l1, score=-0.531, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ................. C=1000, penalty=l1, score=-0.456, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ................. C=1000, penalty=l1, score=-0.422, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ................. C=1000, penalty=l1, score=-0.484, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ................. C=1000, penalty=l2, score=-0.489, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ................. C=1000, penalty=l2, score=-0.531, total=   0.0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ................. C=1000, penalty=l2, score=-0.456, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ................. C=1000, penalty=l2, score=-0.422, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ................. C=1000, penalty=l2, score=-0.484, total=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "[Parallel(n_jobs=1)]: Done  70 out of  70 | elapsed:    0.9s finished\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='warn',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='warn',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
       "                         'penalty': ['l1', 'l2']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.001,0.01,0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss',verbose=5)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4760292852976516\n"
     ]
    }
   ],
   "source": [
    "print(-grid.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd5xU5dn/8c81WylLX+rCgggqSJMFFnsHSyxYAIGoeRKjj6bHqDHRiDXtib9ENKixBRSJLVgRuyJtQUCKKKLI0qX3bdfvjxl0XQaYLcPZmf2+X6/z2pl77jPzPazOtafdt7k7IiIiFYWCDiAiIrWTCoSIiESlAiEiIlGpQIiISFQqECIiElVq0AFqSosWLbxjx45BxxARSSizZ8/+2t2zo72WNAWiY8eOFBQUBB1DRCShmNny/b0W10NMZjbYzJaY2VIzuzHK638zs7mR5VMz21zutcvN7LPIcnk8c4qIyL7itgdhZinAGOAMoBCYZWaT3H3R3j7u/oty/X8C9Ik8bgbcCuQBDsyOrLspXnlFROS74rkH0R9Y6u7L3L0ImACcf4D+w4GnIo8HAVPcfWOkKEwBBscxq4iIVBDPcxDtgBXlnhcCA6J1NLNcoBPw1gHWbRdlvauAqwA6dOhQ/cQikjSKi4spLCxk9+7dQUepFTIzM8nJySEtLS3mdeJZICxK2/4GfhoGPOPupZVZ190fBB4EyMvL06BSIvKNwsJCsrKy6NixI2bRvlLqDndnw4YNFBYW0qlTp5jXi+chpkKgfbnnOcCq/fQdxreHlyq7rojIPnbv3k3z5s3rfHEAMDOaN29e6b2peBaIWUAXM+tkZumEi8Ckip3M7AigKTCtXPNk4Ewza2pmTYEzI20iIjFTcfhWVf4t4lYg3L0EuI7wF/tiYKK7LzSz0WZ2Xrmuw4EJXm7ccXffCNxOuMjMAkZH2mpcaZlz1yuLWbFxZzzeXkQSyNCx0xg6dtrBO9YRcb0Pwt1fcfeu7t7Z3e+MtN3i7pPK9fmDu+9zj4S7P+Luh0eWR+OVcfmGHUyY+RVDHviQBSu3xOtjRKQOatiw4TePBw8eTJMmTTj33HOj9r322mvp3bs33bp1o169evTu3ZvevXvzzDPPVOoz58yZw2uvvVat3HvV+bGYDstuyDPXHEtayLh07DTeXrIu6EgikoSuv/56/v3vf+/39TFjxjB37lxeeeUVOnfuzNy5c5k7dy4XX3xxpT5HBaKGdW2VxfPXHkfH5g344eMFPD3rq6AjiUiSOe2008jKyqrSup999hmDBg2ib9++nHjiiXz66acATJgwgaOPPppevXpxyimnsGvXLkaPHs348eOrtPdRUdKMxVRdrRplMvHqgVwzbjY3PPsxKzfv5hend9FJLpEkcNuLC1m0autB+y1aHe4Ty3mIbm0bcev3ulc7WyyuuuoqHn74YTp37szUqVO57rrreP3117ntttt45513aNWqFZs3b6ZevXrccsstLFiwgHvvvbfan6sCUU7DjFQeuaIfv33uY/7+5mes2ryLu4f0IC1FO1oiEozNmzczffp0Lrroom/aSkpKADjuuOP4/ve/zyWXXMKQIUNq/LNVICpISwnxp4t70rZJPf7fm5+xdutu7h9xDFmZsd99KCK1S6x/6e/dc3j6xwPjGadS3J0WLVowd+7cfV576KGHmDFjBi+99BK9evVi/vz5NfrZ+tM4CjPjF2d05Y8X9eDDzzdw6djprN2q2/VF5NBr2rQpbdq04fnnnwegrKyMefPmAbBs2TLy8/O5/fbbadq0KStXriQrK4tt27bVyGerQBzA0H4d+NfleSzfsIMLx0zl07U1848uInXPCSecwCWXXMKbb75JTk4OkyfHfu/vhAkT+Oc//0mvXr3o3r07L730EgC/+MUv6NGjBz169OD000/n6KOP5tRTT2XevHn06dOn2ieprdz9aQktLy/P4zVh0IKVW7jysVnsLi7lwVF5DOzcPC6fIyI1Z/HixRx11FGVWqc2HmKqSdH+TcxstrvnReuvPYgYHN2uMc9dcyytGmVy+SMzmTRPw0KJJKOnfzwwaYtDVahAxKh9s/o8e/Wx9O7QhJ8+9RFj3/2cZNn7EhGJRgWiEhrXT+OJH/TnnJ5tuPvVT7h10kJKy1QkRCQ56TLXSspMS+Efw/rQtnEmD73/Bau37Obvw/pQLz0l6GgiIjVKexBVEAoZN5/TjT98rxtvLF7L8Iems2H7nqBjiYjUKBWIarjiuE48MKIvi1dv5aIHPuTLr3cEHUlEquPRc8KLACoQ1Tb46NY8+aMBbNlVzJAHPuSjrzYFHUlEaom9w33PnTuXgQMH0r17d3r27MnTTz+9T9/aONy3zkHUgL65zXj2mmO54tFZDH9oOn8f1oczu7cOOpaI1BL169fniSeeoEuXLqxatYq+ffsyaNAgmjRp8k2fMWPGAPDll19y7rnnRh1aIxZz5sxhwYIFDB48uNq5tQdRQw7Lbshz/3ssR7TK4upxs/n3tC+DjiQitUTXrl3p0qULAG3btqVly5asX78+5vU13HcSaNEwg6euyucnT37E7/+7kJWbd/ObQUcQCmnIcJFAvXojrPn44P3WRAa7i+U8ROsecNY9lY4yc+ZMioqK6Ny5c8zraLjvJFE/PZWxo/py66SF/PPdz1m1eRd/vqQnGam6DFakrlu9ejWjRo3i8ccfJxSK7QCOhvtOMqkpIe644GjaNa3Hn15bwrptuxk7Ko/G9TRkuEggYv1Lf++ew5Uv13iErVu3cs4553DHHXeQn58f83oa7jsJmRn/e/Lh/G1oL2Yv38Ql//yQlZt3BR1LRAJQVFTEhRde+M1f+5Wh4b6T2IV9cnj8yv6s3rybIfdPZeGqLUFHEpFDbOLEibz33ns89thj31y+WpmrlDTcdzXFc7jvmvDJmq1c+egstu0u4YGRx3BCl+ygI4kktaoM9x3PQ0y1gYb7rqWObN2I5/73WHKa1uPKR2fxzOzCoCOJSEVXvpy0xaEqVCAOoTaN6zHx6oEMOKwZv/7PPP7+5mcaMlxEai0ViEOsUWYaj17RnyF92vF/Uz7lpuc+pri0LOhYIklJf4B9qyr/FrrMNQDpqSH+emkv2japx31vL2XN1t2MuewYGmTo1yFSUzIzM9mwYQPNmzfHrG7frOrubNiwgczMzEqtp2+kgJgZvx50BG2b1ON3L3zM0Aen8cgV/WiZVblfoIhEl5OTQ2FhYaWGtEhmmZmZ5OTkVGodFYiAXTagA60bZ3Dt+I8Ycv+HPHZlfw5v2TDoWCIJLy0tjU6dOgUdI6HpHEQtcOqRrXj6x/nsLi7logc+ZNaXG4OOJCIS3wJhZoPNbImZLTWzG/fT51IzW2RmC83syXLtpWY2N7JMimfO2qBnThOeu+Y4mjdIZ8TDM3h5/uqgI4lIHRe3AmFmKcAY4CygGzDczLpV6NMFuAk4zt27Az8v9/Iud+8dWc6LV87apEPz+jx7zbH0aNeY656aw8PvLws6kojUYfHcg+gPLHX3Ze5eBEwAzq/Q50fAGHffBODu6+KYJyE0bZDO+B8OYHD31tzx8mJue3EhpWW6VE9EDr14Foh2wIpyzwsjbeV1Bbqa2VQzm25m5adAyjSzgkj7BdE+wMyuivQpSKYrFTLTUrjvsmP4wXGdeHTql1z35Bx2F5cGHUtE6ph4FohoFx5X/FM4FegCnAwMBx42s71z8HWIjA9yGXCvme0zu4a7P+juee6el52dXGMbpYSMW77Xjd+dcxSvLVzDiIdnsGlHUdCxRKQOiWeBKATal3ueA6yK0ue/7l7s7l8ASwgXDNx9VeTnMuAdoE8cs9ZaPzzhMO4bfgwfr9zCRQ98yIqNO4OOJCJ1RDwLxCygi5l1MrN0YBhQ8WqkF4BTAMysBeFDTsvMrKmZZZRrPw5YFMestdo5Pdsw/ocD2LCjiAvvn8r8ws1BRxKROiBuBcLdS4DrgMnAYmCiuy80s9FmtveqpMnABjNbBLwNXO/uG4CjgAIzmxdpv8fd62yBAOjXsRnPXnMsmWkpDB07nbc+WRt0JBFJcpoPIsGs27abHzw2i0WrtnLHBT24bECHoCOJSALTfBBJpGVWJk9fNZATu2bz2+c/5i+Tl2jEShGJCxWIBNQgI5WHv5/HsH7tue/tpfxq4jyKSsoYOnYaQ8dOCzqeiCQJDdaXoFJTQtw9pAftmtTjr1M+Ze223ZSUlZEaUs0XkZqhApHAzIyfnNaFNk3qceOz80lLCXFE66ygY4lIktCfm0ng4r45PHplP/aUlLJo1VY279QNdSJSfSoQSeKELtkc2TqLotIy/lNQGHQcEUkCKhBJJCszjYYZqYyfsZwyDfAnItWkApFkWjXK4MsNO/lg6ddBRxGRBKcCAfDoOeElCTRrkE7zBumMm7486CgikuB0FVMSefrHAwH442ufMPbdz1m1eRdtm9QLOJWIJCrtQZSVwfZ1UFocdJIac1n/Djjw1Myvgo4iIglMBWLTF7DhU9icPIdk2jerzylHtGTCrBUUlZQFHUdEEpQKRPPOkNUGtq+BNQuCTlNjRuXnsn7bHl5ftCboKCKSoFQgABp3gFAqvHYjJMnAdyd2zSanaT2drBaRKlOBAEhJgyYd4Mv34ZOXgk5TI1JCxogBuUxftpHP1m4LOo6IJCAVCIArX4ZrZ0H2UfD676B4d9CJasSleTmkp4S0FyEiVaICsVdKKgy+GzZ9CdPvDzpNjWjeMIOze7TmuTkr2bGnJOg4IpJgVCDK63wKHHE2vP9X2JYcJ3dHDcxl254S/jt3VdBRRCTBqEBUdOYdULIH3hwddJIacUyHphzZOotx05dr5jkRqRQViIqad4b8a2DueFg5O+g01WZmjBqYy6LVW5nz1eag44hIAlGBiObE66FBNrx2U1Jc9npB73bhUV51slpEKkEFIprMRnDaLbBiBix4Nug01dYgI5Uhx7Tjpfmr2bhDkwmJSGxUIPan9who0wum3AJFO4JOU20j83MpKi1jYsGKoKOISIJQgdifUAoMvge2roSpfw86TbV1bZVF/07NNJmQiMRMBeJAco+F7kNg6r2wOfH/8h6Vn8uKjbt497P1QUcRkQSgAnEwZ0Qud33j1mBz1IBB3VvTomGGTlaLSExUIA6mSXs47mfhk9XLpwWdplrSU0MM69eeNz9Zx4qNO4OOIyK1nApELI77GWS1hdduCE8wlMCGD+iAocmEROTgVCBikd4gfKhp9bzwDXQJrF2Tepx6ZCsmFqxgT0lp0HFEpBZTgYhVj4shp394CI7dW4NOUy2jBuby9fYiXluQHONNiUh8xLVAmNlgM1tiZkvN7Mb99LnUzBaZ2UIze7Jc++Vm9llkuTyeOWNiBmfdAzvWwft/CTpNtZxweAtym9dn/HQdZhKR/YtbgTCzFGAMcBbQDRhuZt0q9OkC3AQc5+7dgZ9H2psBtwIDgP7ArWbWNF5ZY9auL/S6DKY/ABuXBZ2mykIhY8SADsz8ciOfrEnsvSERiZ947kH0B5a6+zJ3LwImAOdX6PMjYIy7bwJw93WR9kHAFHffGHltCjA4jlljd9otEEqD138fdJJquaRve9JTQ9qLEJH9imeBaAeUv7usMNJWXlegq5lNNbPpZja4EutiZleZWYGZFaxff4hu/mrUBk78VXhq0mXvHJrPjIOmDdI5t2cbnptTyHZNJiQiUcSzQFiUtopjPKQCXYCTgeHAw2bWJMZ1cfcH3T3P3fOys7OrGbcS8q+FJrnh0V5LE/fLdVR+LjuKSnn+o5VBRxGRWiieBaIQaF/ueQ5QcVqzQuC/7l7s7l8ASwgXjFjWDU5aZnhioXWLYPajQaepst7tm9C9bSPGazIhEYkingViFtDFzDqZWTowDJhUoc8LwCkAZtaC8CGnZcBk4Ewzaxo5OX1mpK32OOp70PEEePsu2Lkx6DRVYmaMys/lkzXbKFi+Keg4IlLLxK1AuHsJcB3hL/bFwER3X2hmo83svEi3ycAGM1sEvA1c7+4b3H0jcDvhIjMLGB1pqz3MYPDdsHszvPvHoNNU2Xm925KVmco4jc8kIhVYshxayMvL84KCgkP/wS/+HOY8Add8CC2PPPSfXwP+MGkh42csZ9pNp9GiYUbQcUTkEDKz2e6eF+013UldXaf+DtIbwuTfJuz0pCPzcykudZ6elfhDmotIzVGBqK4GLeDkG+DzN+Gz14NOUyWHt2zIwMOa8+SMryjVZEIiEqECURP6/Qiadwlf9lqSmHM+jxqYy8rNu3hnybqDdxaROkEFoiakpsOgu2Dj5zDzwaDTVMkZ3VrRMitDJ6tF5BsqEDWl65lw+BnhK5q2J96UnmkpIYb178A7n67XZEIiAlShQJhZyMwaxSNMwht0FxTvhLfvCDpJlQzv356QGeNnaHwmEYmxQJjZk2bWyMwaAIuAJWZ2fXyjJaDsrtD/Kpj9OKyeH3SaSmvTuB6nH9WSiQUr2F2syYRE6rpY9yC6uftW4ALgFaADMCpuqRLZSb+Bek3DJ6wT8LLXUfkd2bijiFcXrA46iogELNYCkWZmaYQLxH/dvZgog+cJ4eJw6u9g+Qew6L9Bp6m0Yzs3p1OLBozTMOAidV6sBWIs8CXQAHjPzHIBzTSzP8dcDi27w5TfQ/GuoNNUyt7JhGYv38SiVfoVi9RlMRUId/+7u7dz97M9bDmRQfYkipTU8DhNm7+CafcFnabSLu6bQ0ZqiHEzdMmrSF0W60nqn0VOUpuZ/cvM5gCnxjlbYjvsJDjyXHj/b7A1sY7nN6mfznm92vLCRyvZurs46DgiEpBYDzH9IHKS+kwgG7gSuCduqZLFmXdAWTG8eVvQSSpt1MBcdhaV8vwcTSYkUlfFWiD2zvB2NvCou88j+qxvUl6zTjDwWpj3FBQGMNJsNfTMaULPnMaM02RCInVWrAVitpm9TrhATDazLKAsfrGSyAm/goat4NUboCyx/slG5ufy2brtzPiidk3FISKHRqwF4n+AG4F+7r4TSCd8mEkOJiMLTrsVVhbAx/8JOk2lfK9nWxppMiGROivWq5jKCM8L/Tsz+wtwrLsn3q3CQek1HNr2gTduhT3bg04Ts3rpKVyS157XFqxh3bbdQccRkUMs1quY7gF+RniYjUXAT83s7ngGSyqhEAz+I2xbDVPvDTpNpYwY0IGSMmeiJhMSqXNiPcR0NnCGuz/i7o8Ag4Fz4hcrCXUYAD0ugQ//AZsS55DNYdkNOf7wFjw54ytKShPrHIqIVE9lRnNtUu5x45oOUiec/gfAYMotAQepnJH5uazaspu3PtFkQiJ1SawF4m7gIzN7zMweB2YDd8UvVpJqnAPH/xwWvQBfTg06TcxOP6olrRtlMk7DgIvUKbGepH4KyAeeiywD3X1CPIMlrWN/Co1y4LUboCwxhtROTQkxvH8H3vt0PV9+vSPoOCJyiBywQJjZMXsXoA1QCKwA2kbapLLS68MZt8Gaj+GjcUGnidmw/u1JCRlPztRehEhdkXqQ1/96gNccjcdUNUdfBDMfgjdHQ/cLILP2n9Jp1SiTQd1bMbFgBb88oyuZaSlBRxKRODvgHoS7n3KARcWhqszgrHtg5wZ4789Bp4nZyPxcNu8s5qX5iTX4oIhUzcH2IAAwsyFRmrcAH7u7Lm2pirZ9oM8ImP5P6HslNO8cdKKDGnhYczpnN2Dc9OVc3Dcn6DgiEmeVGWrjYWBEZHkI+CUw1cw09WhVnXoLpGbC5JuDThITM2Nkfi5zV2xmwcotQccRkTiLtUCUAUe5+0XufhHQDdgDDABuiFe4pJfVCk78NXz6Kix9M+g0MRlyTA710lI0PpNIHRBrgejo7mvLPV8HdHX3jYBmlKmO/GugaSeY/FsoLQk6zUE1rpfG+b3b8sLclWzZpV+9SDKLtUC8b2YvmdnlZnY5MInw3NQNgM3xi1cHpGbAoDth/SdQ8EjQaWIyMj+X3cVlPDenMOgoIhJHsRaIa4FHgd5AH+Bx4Fp33+Hu+52b2swGm9kSM1tqZjdGef0KM1tvZnMjyw/LvVZarn1S5TYrwRxxNnQ6Cd6+E3bW/rkXjm7XmN7tm/BvTSYkktRivZPagQ+At4A3gPf8IN8MZpYCjAHOInzOYriZdYvS9Wl37x1ZHi7Xvqtc+3mx5ExYZjD4HtizFd5JjEFyR+Xnsmz9DqZ9viHoKCISJ7EO930pMBO4GLgUmGFmFx9ktf7AUndf5u5FwATg/OqETWqtukHeD2DWv2Dd4qDTHNQ5PdvQpH4a42boZLVIsor1ENPNhGeTu9zdv0/4y//3B1mnHeFhOfYqjLRVdJGZzTezZ8ysfbn2TDMrMLPpZnZBtA8ws6sifQrWr18f46bUYqfcHJ6B7rWboJYfuslMS+HSvPZMXriWtVs1mZBIMoq1QIQq3BC3IYZ1LUpbxW+9FwlfIdWT8KGrx8u91sHd84DLgHvNbJ87ydz9QXfPc/e87Ozsg25ErVe/GZx8Eyx7G5a8GnSag7qsfwdKy5wJMzWZkEgyirVAvGZmkyMnla8AXgZeOcg6hUD5PYIcYFX5Du6+wd33RJ4+BPQt99qqyM9lwDuET44nv37/Ay2OgNdvhpI9B+8foI4tGnBi12yenLmcYk0mJJJ0Yj1JfT3wINAT6AU86O4Hu0FuFtDFzDqZWTowjPDlsd8wszblnp4HLI60NzWzjMjjFsBxhKc6TX4paTD4Lti4DGb8M+g0BzUqP5e1W/fw5uK1B+8sIgklprGYANz9WeDZSvQvMbPrgMlACvCIuy80s9FAgbtPIjy39XlACbARuCKy+lHAWDMrI1zE7nH3ulEgAA4/HboOhnf/DL2GQ8OWQSfar1OPbEnbxpmMm/4Vg49uc/AVRCRh2IGuVjWzbex73gDC5xfc3RvFK1hl5eXleUFBQdAxas7XS+H+fOg1DM6/L+g0B3TfW5/xl9c/5a1fncRh2Q2DjiMilWBmsyPne/dxsOG+s9y9UZQlqzYVh6TU4nAY8OPwpEKr5gad5oAu7dee1JAxXlOSiiSVWE9SSxBO+g3Ubw6v3VirL3ttmZXJ4KNb85+CFewqSoxpVEXk4FQgarPMxnDa7+GrabDw+aDTHNDI/Fy27i7hxfmrDt5ZRBKCCkRt12cUtOoBU26B4l1Bp9mvAZ2a0aVlQw0DLpJEVCBqu1BKeHrSLSvgw38EnWa/zIxRA3OZX7iFeSs0wK9IMlCBSAQdj4du58MHf4MtK4NOs18X9mlH/XRNJiSSLFQgEsUZt0NZKbzxh6CT7FdWZhoX9GnHpHmr2LyzKOg4IlJNKhCJomkuHPsT+HgirJgZvc+j54SXAI0ckMuekjKema3JhEQSnQpEIjn+F9CwNbx6A5TVzrGPurVtRN/cpoyf8RVlZbX30lwROTgViESS0RDOuA1WzYH5TwedZr9G5efyxdc7mPr510FHEZFqUIFIND0uhXZ9w+ci9mwPOk1UZ/VoTbMG6TpZLZLgVCASTSgEg/8I29fAB/8XdJqoMlLDkwlNWbSW1Vtq770bInJgKhCJqH0/6DkUPrwPNn0ZdJqoRgzogANPaTIhkYSlApGoTv9D+Ca61w8282sw2jerz8lds5kw8ytNJiSSoFQgElWjtnD8L2HxJPji/aDTRDUyP5d12/YwZZEmExJJRCoQiezY66Bxh/Bor2W1bxTVk49oSbsm9fj3NJ2sFklEKhCJLK0enDka1i6AOY8HnWYfKSFjRH4Hpi3bwNJ124KOIyKVpAKR6LpdALnHwVt3QFlJ0Gn2cWlee9JSjHHTNZmQSKJRgUh0ZjD4bti5ETbXvi/hFg0zOLtHG56dXcjOotpXwERk/1QgkkGbXnDM92HbaijeGXSafYzMz2XbnhImzdVkQiKJRAUiWZz6e7AQbPyi1k1PmpfblCNbZ/HEtOV4LcsmIvunApEsGmZDkw6wexM8ejas+TjoRN8wM0bm57Jo9VY+0mRCIglDBSKZZLWFZofD10tg7Inw0i/D5yZqgQv6tKOBJhMSSSgqEMnEDLJaw09mQ78fwezH4B/HwKyHA79PomFGKkOOyeGl+avZtEOTCSWqhXcdz8K7jg86RrUly3ZAfLdFBSIZ1WsKZ/8Jrn4fWh0NL/8Kxp4Eyz8MNNbI/FyKSsr4z+y6NT5TMn0ZSd2iApHMWnWHy1+ESx6DXZvg0bPgmf+BrcFcTXRE6yz6d2zGuOmaTEgkEahAJDsz6H4hXDcTTvwNLH4R/pEH7/8VSvYc8jgj8jvw1cadvPfZ+gP2Gzp2GkPHTjtEqUQkGhWIuiK9AZx6M1w7AzqfAm+OhjEDYMlrhzTG4KNb06Jh+kHvrL5lw/XcsuH6Q5RKRKJRgahrmnWCYeNh5HMQSoWnhsL4S+DrpYfk4zNSUxjarz1vfbKWlZs1mZBIbaYCUVcdfhpc8yGceScsnwb358OUW2FP/AfVG94/MpnQjNo3NIiIfCuuBcLMBpvZEjNbamY3Rnn9CjNbb2ZzI8sPy712uZl9Flkuj2fOOis1PTxk+E9mQ89LYeq94fMT8yfG9W7snKb1Oe3IlkyY9RVFJZpMSKS2iluBMLMUYAxwFtANGG5m3aJ0fdrde0eWhyPrNgNuBQYA/YFbzaxpvLLWeVmt4IL74X/egEZt4LkfwSODYfW8uH3kiPxcvt5exOSFa+L2GSJSPfHcg+gPLHX3Ze5eBEwAzo9x3UHAFHff6O6bgCnA4DjllL3a94MfvgXn/QM2LA3fO/Hiz2HHhhr/qJO6ZNO+WT3+rTurRWqteBaIdkD5O6IKI20VXWRm883sGTNrX5l1zewqMysws4L16w982aTEKBQKjwz7k9kw4GqY80T4buyZD0FpzQ3XHQoZIwfkMvOLjSxZo8mERGqjeBYIi9JW8cD2i0BHd+8JvAHsnRYtlnVx9wfdPc/d87Kzs6sVViqo1wTOugeumQptesIrv4YHT4IvP6ixj7gkrz3pqSHGz9BehEhtFM8CUQi0L/c8B/jOLbzuvsHd996t9RDQN9Z15RBpeRR8fxJc+gTs3gKPnQP/uRK2FFb7rZs1SOfcHm14bs5KduzRZEIitU08C8QsoIuZdTKzdGAYMJo8HscAAAzYSURBVKl8BzNrU+7pecDiyOPJwJlm1jRycvrMSJscyJUvh5eaZgbdzodrZ8JJN8KSV+C+fvDen6F4d7XeekR+Ltv3lPDC3JU1FFZEakrcCoS7lwDXEf5iXwxMdPeFZjbazM6LdPupmS00s3nAT4ErIutuBG4nXGRmAaMjbRKk9Ppwyk3hQnH4aeF5sO8fAJ+8UuXLYo/p0IRubRrxb00mJFLrxPU+CHd/xd27untnd78z0naLu0+KPL7J3bu7ey93P8XdPym37iPufnhkeTSeOaWSmubC0HEw6gVIyYAJw2H8xfD1Z5V+q72TCX2yZhuzl2+KQ1gRqSrdSS1V1/mU8EnsQXfDipnhu7Ff/x3s3lqptzm/d1uyMlI1mZBILaMCIdWTkgYD/zd8WWyvYfDhP+C+PJg3Acpiu0u6QUYqF/XN4ZWP17Bh+6EfYVZEolOBkJrRsCWcPyZ8o13jHHj+x/DIIFj1UUyrjxjQgaLSMiYWVP/qKBGpGSoQUrNy+oaH7Dj/ftj0BTx4Ckz6Kez4+oCrdWmVRf5hzRg/YzmlmkxIpFZQgZCaFwpBnxHhw04Dr4W548N3Y88Ye8C7sUfm51K4aRfvfaq74kVqAxUIiZ/MxjDozvCw4m37wKu/gbEnwBfvRe1+ZrfWZGdlaHwmkVpCBULiL/uI8CWxQ8dB0XZ4/Hsw8XLYvOI73dJTQwzv1563l6xjTVnjgMKKyF4qEHJomMFR3wvfZHfKzfDp5PDd2O/+6Tt3Yw/r3wEDXivqHVxWEQFUIORQS6sHJ/0GrpsJXQfB23fCmP6w+CVwp22Tepx+VCsmF/em2FOCTitSp6lASDCadIBLHw8PBJhWH54eAeOGwPpPGZmfy1avz9SSI4JOKVKnpQYdQOq4w06Cqz+AWQ/D23fBAwM5of/VHB7K5YFdZzL5vg/ITEuh3t4lPeXb5+kh6qVFnqd/2ycz/bv9K/ZJCUUbTV5EKlKBkOClpEL+1dDjYnjzNmz6GF5MT+fVsgGUlHShaA/sKTOKSp09pbCn1Cgqhc0lzteEKMNwjFLf+zhEaaQ9vITK/QwRCoVITU0hLTWVtNRUUlNTSUtLJT01vKRGHmekpZKelkZaWioZqamkp6WSkZ5Gxt6f6alkpqWRnpZGZkYamWmp1EtPJzMjlYy0NCyUAhb6diDDkiLAI8+j/Syr0MZ++7iXUeaOl4V/lrlDmeOUUVbmlHlp+HnkNS8rwyPrebl+37QTbi/z8Ge4+3eel3kZlDkbSjIBWDprSjhelF+n+3fvoI8ykUvFhgovH/w+mIpd9l1n3/co32VNcf1w2weT9umXaNYWNyDF4jO3uyXLCJp5eXleUFAQdAypCSvnsPPhc6jvO4NOIpIQFpd14KjRH1dpXTOb7e550V7THoTUPu2O4Yu0zpiX0e36NyJ/NZd++xd22d7HZRVeKwuP/xS1vTS8btT2sn2Xcu1lZaWUlJRSVFJCUXExxSUlFJeUUlxSTElxKcWlJZRE2kpKSigpLaG0pJSS0lJKS0vYvnIxjpHZqiuYYWaAha/sijx3Qt+0m5V7DQvvhRiYhcLt3/TZu863r32n3ULh7hgWqvj+ocg64XUtsm759cKvh77Tvv69fwFOq5N/9M2vy6JNAGkHPoxn9t3Tn/v0jvaWFRsrfMY+q0Q5lLj3PVZP+X8AtDnz5wfMWVPieVBzzev3Ekot46g4vLcKhNRabqHwHBQBCwHpkaUqFt51PADdr/1rTUUKzMLpfwOg+8lDAk5STe/cBUD3gWcFHKT6/O074/beuopJRESiUoEQEZGoVCBERCQqFQgREYlKBUJERKJSgRARkahUIEREJCoVCBERiUoFQkREolKBEBGRqFQgREQkKhUIERGJSgVCRESiUoEQEZGoVCBERCSquBYIMxtsZkvMbKmZ3XiAfhebmZtZXuR5RzPbZWZzI8s/45lTRET2FbcJg8wsBRgDnAEUArPMbJK7L6rQLwv4KTCjwlt87u6945VPREQOLJ57EP2Bpe6+zN2LgAnA+VH63Q78CdgdxywiIlJJ8SwQ7YAV5Z4XRtq+YWZ9gPbu/lKU9TuZ2Udm9q6ZnRDtA8zsKjMrMLOC9evX11hwERGJb4GINk+3f/NieNbyvwG/itJvNdDB3fsAvwSeNLNG+7yZ+4PunufuednZ2TUUW0REII7nIAjvMbQv9zwHWFXueRZwNPCOmQG0BiaZ2XnuXgDsAXD32Wb2OdAVKIhjXqlFurdpHHQEkTovnnsQs4AuZtbJzNKBYcCkvS+6+xZ3b+HuHd29IzAdOM/dC8wsO3KSGzM7DOgCLItjVhERqSBuexDuXmJm1wGTgRTgEXdfaGajgQJ3n3SA1U8ERptZCVAKXO3uG+OVVWqhK18OOoFInRfPQ0y4+yvAKxXabtlP35PLPX4WeDae2URE5MB0J7WIiESlAiEiIlGZux+8VwLIy8vzggJd5CQiUhlmNtvd86K9pj0IERGJSgVCRESiUoEQEZGoVCBERCQqFQgREYlKBUJERKJSgRARkahUIEREJCoVCBERiSpp7qQ2s/XA8mq8RQvg6xqKE6Rk2Q7QttRWybItybIdUL1tyXX3qDOuJU2BqC4zK9jf7eaJJFm2A7QttVWybEuybAfEb1t0iElERKJSgRARkahUIL71YNABakiybAdoW2qrZNmWZNkOiNO26ByEiIhEpT0IERGJSgVCRESiUoGIMLPbzWy+mc01s9fNrG3QmarKzP5sZp9Etud5M2sSdKaqMrNLzGyhmZWZWcJdkmhmg81siZktNbMbg85THWb2iJmtM7MFQWepDjNrb2Zvm9niyH9bPws6U1WZWaaZzTSzeZFtua1G31/nIMLMrJG7b408/inQzd2vDjhWlZjZmcBb7l5iZn8EcPcbAo5VJWZ2FFAGjAV+7e4JM6+smaUAnwJnAIXALGC4uy8KNFgVmdmJwHbgCXc/Oug8VWVmbYA27j7HzLKA2cAFifh7MTMDGrj7djNLAz4Afubu02vi/bUHEbG3OEQ0ABK2crr76+5eEnk6HcgJMk91uPtid18SdI4q6g8sdfdl7l4ETADODzhTlbn7e8DGoHNUl7uvdvc5kcfbgMVAu2BTVY2HbY88TYssNfbdpQJRjpndaWYrgBHALUHnqSE/AF4NOkQd1Q5YUe55IQn6RZSszKwj0AeYEWySqjOzFDObC6wDprh7jW1LnSoQZvaGmS2IspwP4O43u3t7YDxwXbBpD+xg2xLpczNQQnh7aq1YtiVBWZS2hN0zTTZm1hB4Fvh5hSMICcXdS929N+EjBf3NrMYO/6XW1BslAnc/PcauTwIvA7fGMU61HGxbzOxy4FzgNK/lJ5oq8XtJNIVA+3LPc4BVAWWRciLH658Fxrv7c0HnqQnuvtnM3gEGAzVyIUGd2oM4EDPrUu7pecAnQWWpLjMbDNwAnOfuO4POU4fNArqYWSczSweGAZMCzlTnRU7s/gtY7O7/F3Se6jCz7L1XKZpZPeB0avC7S1cxRZjZs8ARhK+YWQ5c7e4rg01VNWa2FMgANkSapifwFVkXAv8AsoHNwFx3HxRsqtiZ2dnAvUAK8Ii73xlwpCozs6eAkwkPLb0WuNXd/xVoqCows+OB94GPCf//DvBbd38luFRVY2Y9gccJ//cVAia6++gae38VCBERiUaHmEREJCoVCBERiUoFQkREolKBEBGRqFQgREQkKhUIkUows+0H73XA9Z8xs8Mijxua2Vgz+zwyEud7ZjbAzNIjj+vUjaxS+6hAiBwiZtYdSHH3ZZGmhwkPftfF3bsDVwAtIgP7vQkMDSSoSIQKhEgVWNifI2NGfWxmQyPtITO7P7JH8JKZvWJmF0dWGwH8N9KvMzAA+J27lwFERn19OdL3hUh/kcBoF1akaoYAvYFehO8snmVm7wHHAR2BHkBLwkNJPxJZ5zjgqcjj7oTvCi/dz/svAPrFJblIjLQHIVI1xwNPRUbSXAu8S/gL/XjgP+5e5u5rgLfLrdMGWB/Lm0cKR1FkQhuRQKhAiFRNtKG8D9QOsAvIjDxeCPQyswP9P5gB7K5CNpEaoQIhUjXvAUMjk7VkAycCMwlP+XhR5FxEK8KD2+21GDgcwN0/BwqA2yKji2JmXfbOgWFmzYH17l58qDZIpCIVCJGqeR6YD8wD3gJ+Ezmk9CzheSAWEJ5HewawJbLOy3y3YPwQaA0sNbOPgYf4dr6IU4CEG11UkotGcxWpYWbWMDKJfHPCexXHufuayHj9b0ee7+/k9N73eA64KYHn45YkoKuYRGreS5FJXNKB2yN7Frj7LjO7lfC81F/tb+XI5EIvqDhI0LQHISIiUekchIiIRKUCISIiUalAiIhIVCoQIiISlQqEiIhE9f8BB3GBwGu94pEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "#train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "#train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "#train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "#train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    #plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的logloss。\n",
    "可以看出在训练集上C越大（正则越少）的模型性能越好；\n",
    "但在测试集上当C=1时性能最好（L1正则）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "保存模型用于测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(grid.best_estimator_, open(\"diabetes_L1_org.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 换正确率做评价指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 14 candidates, totalling 70 fits\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................. C=0.001, penalty=l1, score=0.649, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................. C=0.001, penalty=l1, score=0.649, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................. C=0.001, penalty=l1, score=0.649, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................. C=0.001, penalty=l1, score=0.654, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................. C=0.001, penalty=l1, score=0.654, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................. C=0.001, penalty=l2, score=0.734, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................. C=0.001, penalty=l2, score=0.714, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................. C=0.001, penalty=l2, score=0.753, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................. C=0.001, penalty=l2, score=0.784, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................. C=0.001, penalty=l2, score=0.752, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l1, score=0.682, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l1, score=0.682, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l1, score=0.734, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .................. C=0.01, penalty=l1, score=0.725, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l1, score=0.712, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l2, score=0.753, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l2, score=0.708, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l2, score=0.773, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l2, score=0.804, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l2, score=0.771, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l1, score=0.760, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l1, score=0.740, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l1, score=0.773, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l1, score=0.784, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l1, score=0.758, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l2, score=0.766, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l2, score=0.747, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l2, score=0.792, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l2, score=0.797, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l2, score=0.771, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ..................... C=1, penalty=l1, score=0.760, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ..................... C=1, penalty=l1, score=0.747, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ..................... C=1, penalty=l1, score=0.786, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ..................... C=1, penalty=l1, score=0.797, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ..................... C=1, penalty=l1, score=0.771, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ..................... C=1, penalty=l2, score=0.760, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ..................... C=1, penalty=l2, score=0.740, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ..................... C=1, penalty=l2, score=0.786, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ..................... C=1, penalty=l2, score=0.797, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ..................... C=1, penalty=l2, score=0.771, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] .................... C=10, penalty=l1, score=0.760, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] .................... C=10, penalty=l1, score=0.740, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] .................... C=10, penalty=l1, score=0.786, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] .................... C=10, penalty=l1, score=0.797, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] .................... C=10, penalty=l1, score=0.771, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] .................... C=10, penalty=l2, score=0.760, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] .................... C=10, penalty=l2, score=0.740, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] .................... C=10, penalty=l2, score=0.786, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] .................... C=10, penalty=l2, score=0.797, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] .................... C=10, penalty=l2, score=0.771, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ................... C=100, penalty=l1, score=0.760, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ................... C=100, penalty=l1, score=0.740, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ................... C=100, penalty=l1, score=0.786, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ................... C=100, penalty=l1, score=0.797, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ................... C=100, penalty=l1, score=0.771, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ................... C=100, penalty=l2, score=0.760, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ................... C=100, penalty=l2, score=0.740, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ................... C=100, penalty=l2, score=0.786, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ................... C=100, penalty=l2, score=0.797, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ................... C=100, penalty=l2, score=0.771, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .................. C=1000, penalty=l1, score=0.760, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .................. C=1000, penalty=l1, score=0.740, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .................. C=1000, penalty=l1, score=0.786, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .................. C=1000, penalty=l1, score=0.797, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .................. C=1000, penalty=l1, score=0.771, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .................. C=1000, penalty=l2, score=0.760, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .................. C=1000, penalty=l2, score=0.740, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .................. C=1000, penalty=l2, score=0.786, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .................. C=1000, penalty=l2, score=0.797, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .................. C=1000, penalty=l2, score=0.771, total=   0.0s\n",
      "0.7747395833333334\n",
      "{'C': 0.1, 'penalty': 'l2'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  70 out of  70 | elapsed:    0.8s finished\n"
     ]
    }
   ],
   "source": [
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.001,0.01,0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "#缺省scoring为正确率\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5,verbose=5)\n",
    "grid.fit(X_train,y_train)\n",
    "lr_best=grid.best_estimator_\n",
    "print(grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXwV9b3/8dcnISHsS1hkJ0BQFgE1gopLwY2qxeJSodW2tlfsr+r1eqtWba8LVlv1tra99d661FqtFRFcUFDcd0UQWcMWQCCAEHbCFpJ8fn/MiYZwAodwJpPl/Xw8ziMzc74z5zMRzyffz8x8v+buiIiIVJQSdQAiIlIzKUGIiEhcShAiIhKXEoSIiMSlBCEiInEpQYiISFyhJggzG2Fmi80sz8xuifN+VzN7x8y+MLO5ZnZeufduje232MzODTNOERE5kIX1HISZpQJLgLOBfGAGMMbdc8u1eQT4wt3/z8z6AlPdvXts+RlgMNAReBPo7e4loQQrIiIHCLMHMRjIc/fl7l4EjAcurNDGgeax5RbA2tjyhcB4d9/r7iuAvNjxRESkmjQI8didgNXl1vOBIRXa3Am8bmbXAU2As8rt+2mFfTsd7MPatGnj3bt3P4JwRUTqn88//3yju7eN916YCcLibKtYzxoDPOHuvzezk4GnzKx/gvtiZmOBsQBdu3Zl5syZRxiyiEj9YmYrK3svzBJTPtCl3HpnvikhlfkpMAHA3T8BMoA2Ce6Luz/i7jnuntO2bdwEKCIiVRRmgpgBZJtZlpmlA6OByRXarALOBDCzPgQJoiDWbrSZNTSzLCAb+CzEWEVEpILQSkzuXmxm1wLTgFTgcXdfYGbjgJnuPhn4BfComd1AUEL6sQe3VS0wswlALlAMXKM7mEREqldot7lWt5ycHNc1CBEps2/fPvLz89mzZ0/UodQIGRkZdO7cmbS0tP22m9nn7p4Tb58wL1KLiEQmPz+fZs2a0b17d8zi3fdSf7g7mzZtIj8/n6ysrIT301AbIlIn7dmzh8zMzHqfHADMjMzMzMPuTSlBiEidpeTwjar8LpQgRERiLnv4Ey57+JOow6gxlCDqkr+fH7zqgrp0LlJvNW3a9OvlESNG0LJlSy644IK4ba+55hoGDRpE3759adSoEYMGDWLQoEFMnDjxsD5z1qxZvPbaa0cUdxldpBYRqQY33XQTu3bt4uGHH477/kMPPQTAl19+yQUXXMDs2bOr9DmzZs1i/vz5jBgxosqxllEPQkSkGpx55pk0a9asSvsuXbqUc889lxNOOIHTTz+dJUuWADB+/Hj69+/PwIEDGTZsGLt372bcuHE8/fTTVep9VKQehIjUeXe9vIDctdsP2S53XdAmkesQfTs2547v9Dvi2BIxduxYHnvsMXr27MlHH33Etddey+uvv85dd93Fu+++S/v27dm6dSuNGjXi9ttvZ/78+fzxj3884s9VghAJW9m1lCunRBtHEiy491QA+t32YcSRHJnSol0ApKQ3jjiSQ9u6dSuffvopF1988dfbiouLARg6dCiXX3Yxoy44l9E/uSbpn60EIRKyBeu2AVA9f2tKPBX/0t+9bhEAjTocs9/2sp7Ds1efXD2BJcDdadOmTdxrEo8++ijvT3mWqW++y8CBA5k7d25SP1vXIEREarBWrVrRoUMHXnjhBQBKS0uZM2cOAMuXL2fwCYO44+bradWqFWvWrKFZs2bs2LEjKZ+tBCEiUg1OO+00Lr30Ut566y06d+7MtGnTEt53/Pjx/PWvf2XgwIH069ePV155BYAbbriBE4eP5MThIznrrLPo378/w4cPZ86cORx33HG6SC0iUlMVFhZ+vfzBBx8ktE/37t2ZP3/+ftt69OgRN6FMnjz5gHJZ27ZtkzZ5mhKEiEhMTbr2UBOoxCQiInEpQYiISFxKECIiEpcShIiIxKUEISJSRqMI7yfUBGFmI8xssZnlmdktcd5/0Mxmx15LzGxruffuN7MFZrbQzP5smvlDRGqZsuG+Z8+ezcknn0y/fv0YMGAAzz777AFt69Vw32aWCjwEnA3kAzPMbLK755a1cfcbyrW/DjgutnwKMBQYEHv7Q+AM4N2w4hURCUvjxo158sknyc7OZu3atZxwwgmce+65tGzZ8us29W2478FAnrsvd/ciYDxw4UHajwGeiS07kAGkAw2BNGB9iLGKiISmd+/eZGdnA9CxY0fatWtHQUFBwvsfbLjvnGHfYchZ3611w313AlaXW88HhsRraGbdgCzgbQB3/8TM3gHWAQb8xd0XxtlvLDAWoGvXrkkNXkTqkFdvga/mfb2aHhvNlYqjuX4VG+wukesQRx0L3/7dYYfy2WefUVRURM+ePRPe52DDfb864e+0b9uGvY2OqlXDfce7ZuCVtB0NTHT3EgAz6wX0ATrH3n/DzE539/f3O5j7I8AjADk5OZUdW0SkRli3bh1XXHEF//jHP0hJSayAc6jhvq/691tq5XDf+UCXcuudgbWVtB0NlD+7UcCn7l4IYGavAicB78fZV0Tk4Cr8pV9UyXDfYc7dsX37ds4//3x+85vfcNJJJyW8X10d7nsGkG1mWWaWTpAEJldsZGZHA62A8lM4rQLOMLMGZpZGcIH6gBKT1F0L1m37eh4FkdquqKiIUaNG8cMf/pBLL730sPatk8N9u3sxcC0wjeDLfYK7LzCzcWY2slzTMcB4dy9fIpoILAPmAXOAOe7+clixioiEacKECbz//vs88cQTX9++ejh3KdXJ4b7dfSowtcK22yus3xlnvxLg6jBjExEJW9lw35dffjmXX355QvtouG+RQzAvJZUSKC2BlNSow5H6og7MG55MShBSc5SWwJcfwLznOHrfQlIphbvbQNP20KwDNO8Y+9kBmnXc/2fDZlFHL1LnKEFItNxh7SyYNxHmT4LC9ZDejO0pLdhjjegw9AewfR3sWAublgUJZE+ci9fpzWIJo3wiqZBQmrZTb6SecXc0Sk9g/8u8iVGCkGhsXArzngtem5dDajr0PheOvRSyz2HtA2cD0GH4rw/ct2gX7FgH29fG/7niAyj8CkqL99/PUoPeSKWJJPazYdNq+AVI2DIyMti0aROZmZn1Pkm4O5s2bSIjI+Ow9lOCkOqzfW3QS5j3HKybA5YCWafDab+AYy6ARi0PfQwInn7N7Bm8KlNaCjsLgp5HWQ9k+7pvEsmmvCCR7I3TG2nYPE4pq0IiadL2kL2R3UUlzFq1hSl7TmU36XSatpiMtBQy0lJpmJZKRoNgOXjFlhuklmvzzba0VKv3X3KHq3PnzuTn58cd0mLftq8ASNta+5+vTfRcMjIy6Ny580HbVKQEIeHatRkWTg5KSF9+CDh0OgFG/A76jYJmR4XzuSkp0Kx98Op4XOXt9hbCjq/iJJLYz43vBe8HD/mXO36DctdGgkSyt3F7lu9tzpytjfhwQzrvf9WA7SUNSWEo6RRT9G4epVX8PkoxvkkmDcolmbSU/ZJKWbJp2KBi4qnwfoVklLHfsVJp2CCFlJTanZDS0tLIysqK+96Ce68CoM9tH1ZnSKEI81yUICT5inbBkleDpLD0DSjdB5nZMOw26H/xwf/yr24Nm0LDXtCmV+VtSkugcMMBvZCirWsoLFhF6Yq5NN77Jo19N30IxogZDZAG+xo3p3hfEW5Go459cQ+6+6XulH69TNz1/ZfL2sSWi53SfRX2LwWn3D6lXunYNuXtjb0q9qXMIMWCnktKbDll3x4MZ8lvTqza77uGSC3aDVDrzwOCc9lIgr3vw6QEAaE+Xl9vlOyD5e8G5aNFU6CoMCjPnPSz4LrCUQOCb5zaKCUVmndga4NMPtvaiekbNjN9xSZy126n1CE9NYWBXVpwatcMhrYron+znWTsDhJK2vZ17J75LEYp1qjV1wOUVdel8rLEUuJOaek3yaOk9JskVVoaez+WZOK9X+pOcSns27cFBxo0aFFNZxCO4qIgddb284DgXIotnK9yJQiputJSyP8sSAoLXoBdmyCjZdBLOPZS6DY0KPXUUpsK9/LZis1MX7GZT5dvYvH6HbhDwwYpHNe1JdcNz2ZIj9Yc37UVGWmVf+WvnvMpAP0un1RdoX/NCJJRshLSgntPBaDfLW8m6YjRqCvnAcG5tKX40A2rQAlCDt/6BbE7kCbBtlXQoBEc/e0gKfQ6Exo0jDrCKtmwYw/Tlwe9g+nLN7N0Q/AUbKO0VE7o1orzj+3AkB6ZDOzSgoYNdLus1H1KEJKYLSth/sTgusKG3OCW0Z7DYfiv4ZjzauWDauu27d4vISzfuBOAJump5HRvzajjOzEkK5NjO7UgvUHt7QmJVJUShFSusAByXwx6C6unB9u6nATn/XdwB1KTNtHGd5hWb97F9BWbmb58E9NXbGbV5mDSmGYZDRjcvTWjB3dhSFYm/To2p0GqEoKIEoTsb++O4CLzvOdg2TvB7Z3t+sGZdwTXFlp1izrChLg7Kzft+rp3MH3FZtZsDe5cadk4jcHdW/OjU7ozJKs1fTo0J7WW39IpEgYlCIHivZD3ZpAUFr8GxbuhRVcYej0cewm07xd1hIfk7iwr2FkuIWxi/fa9AGQ2SWdIj9aMPb0HQ3q0pne7ZrX+Hn+R6qAEUV+VlsDKj4KkkPtSML5R40w47vLgYnOXwTX6ttTSUmfphsL9eggbC4OE0K5ZQ4b0yGRIVmtO6tGanm2b6ilkkSpQgqhP3IMhLuY9B/OfDx78Sm8aDHNx7KXQ4wxITYs6yrhKS52FX23/unfw2YrNbNm1D4COLTI4LbsNQ7JaM6RHJt0zGyshiCSBEkR9sGlZcPfRvOdg01JISYPsc+DYe6D3iGBsoxqm1GFZ6VF8/P7yrxPC9j3Bvd5dWjfizD7tYz2ETDq3aqSEIBICJYi6asdXQS9h3nPBcNoYdD8VTrkO+o6ERq2ijvCgfr97JO8W94OpC8lq04Tzju3AkB6tGZKVSceWjaIOT6ReCDVBmNkI4E8ED3I+5u6/q/D+g8Cw2GpjoJ27t4y91xV4DOgCOHCeu38ZZry1Xmkx7NoI/xgJK94HHDoMgnPugf4XBaOQ1gJT5q7j3eJ+jEqfzi033kb75oc3RLGIJEdoCcLMUoGHgLOBfGCGmU1299yyNu5+Q7n21wHlh918ErjH3d8ws6ZAaVix1mrusOoTmPVUMOyFlwavM34Z3IHUJjvqCA/LpsK93P7SfLJT1nFlw3do33xc1CGJ1Fth9iAGA3nuvhzAzMYDFwK5lbQfA9wRa9sXaODubwC4e2GIcdZOO9bDnH/BF/8M5jZIbwZN2gVDUF/9Xo2+A+lg7nw5l+179jGu0SukWu0fq1+kNgszQXQCVpdbzweGxGtoZt2ALODt2KbewFYzez62/U3gFveKg/LXMyXFsPR1+OIpWDIteIit68lw6n9Cv+/C098L2tXS5PDa/K94ec5afnF2b7pP3xh1OCL1XpgJIt63VGV/Eo4GJpZLAA2A0whKTquAZ4EfA3/b7wPMxgJjAbp27XrkEddUG/OCpDDnmWDO5ibt4JRr4bgral0JqTJbdhbx6xfn069jc372rZ4smR51RCISZoLIJ7jAXKYzsLaStqOBayrs+0W58tSLwElUSBDu/gjwCEBOTk7dqkcU7QweYJv1FKz6OBgcr/e5wYNs2efU2OcVqmrcK7ls3VXEkz8ZTJrGQRKpEcJMEDOAbDPLAtYQJIHvV2xkZkcDrYBPKuzbyszaunsBMByYGWKsNYM7rJkFXzwZDKVdtANa94Sz7oSBY8KbnjNib+au54Uv1nD9mdn07dg86nBEJCa0BOHuxWZ2LTCN4DbXx919gZmNA2a6++RY0zHAeHf3cvuWmNmNwFsWPAH1OfBoWLFGbucmmPtsUEbakBvMr9Dvu0EJqdsptfaaQiK27drHbS/M45ijmnHNsINM+yki1S7U5yDcfSowtcK22yus31nJvm8AA0ILLmqlJbD8naCEtHgqlBRBx+PhggeDUVMzav9UiIm4e0oum3YW8fiPT9ScCyI1jJ6krm5bVsLsp+GLp2F7fvBEc85P4fgrasWoqcn0zuINTPw8n2uH9aJ/p/0T4rjMB4Dg7gQRiYYSRHXYtwcWvRKUkJa/F2zrOQzOuRuOOb/WTtF5JLbv2cetk+bRu31TrjtTpSWRmkgJIkxfzQtKSPMmwO4twRwL37oFBn0fWtbh23ITcO+UhWzYsYeHrxiq+Z1FaigliGTbvTWYu3nWU7BuNqSmB8NpH38FZH0LUlRnf39JAeNnrOZnZ/RkYJeWUYcTun4d6sf1JKl7lCCSwR2+/DAoIeW+BMV7oH1/GHEfDPgeNG4ddYQ1RuHeYm59fh492zbhP86qGw/5idRVShBHYvu62AXnf8KWFdCweVA+Ou4K6Hhcnb49tap+O3Uha7ftZuLPTiEjTaUlkZpMCeJwleyDJa8FJaS8N4KRU7udGlxb6DOyRk6+U1N8nLeRp6ev4qrTsjihW82ej0JElCASV7AkeMJ5znjYWQBNj4Kh1we9hcyeUUdX4+3cW8zNk+aS1aYJvzjn6KjDqV5XTok6gqTpd9uHUYeQFHXlPCDcc1GCOJi9hbDgheDawurpsfGQRgQXnHudDan69SXq/tcWsWbrbiZcfbJKSyK1hL7hKnKH/Bkw68kgORQVQmYvOOuu2HhI7aOOsNb5dPkm/vHJSq4c2p0Tu+uCvUhtoQRRpmQffPyXoLdQsAjSGkO/UUEJqetJuuBcRbuLSvjlpLl0bd2Ym86tZ6UlkVpOCWL7OihYCLs2Q/506JQD3/kT9LsIMjSy6JF6YNpiVm7axTNXnUTj9MT/uT179ckhRiUiiVCCaNQymHuhWQe44nlo1yfqiOqMmV9u5u8fr+CKk7pxcs/MqMMRkcOkBJHWCDqeEJSQlBySZs++Em6eOJdOLRtxy7ePiTocEakCJQjQ9YUQ/OGNJSzfuJOn/20ITRrqn5lIbaSBgSTpZq3awmMfLGfM4K4M7dUm6nBEpIqUICSp9uwr4abn5nBU8wxuO0+lJZHaTH1/Sao/vbWUZQU7+cdPBtMsIy3qcETkCITagzCzEWa22MzyzOyWOO8/aGazY68lZra1wvvNzWyNmf0lzDglOeas3srD7y3jezmdOaN326jDEZEjFFoPwsxSgYeAs4F8YIaZTXb33LI27n5DufbXAcdVOMzdwHthxSjJs7e4hJsmzqFdswx+dX7fqMMRkSQIswcxGMhz9+XuXgSMBy48SPsxwDNlK2Z2AtAeeD3EGCVJ/vJ2HkvWF/Lbi46lRSOVlkTqgjATRCdgdbn1/Ni2A5hZNyALeDu2ngL8HrgpxPgkSeav2cb/vruMi47vxLBj2kUdjogkSZgJIt7DBV5J29HARHcvia3/HJjq7qsraR98gNlYM5tpZjMLCgqOIFSpqqLiUm6aOJfMJunccUG/qMMRkSQK8y6mfKBLufXOwNpK2o4Grim3fjJwmpn9HGgKpJtZobvvd6Hb3R8BHgHIycmpLPlIiP733TwWrtvOoz/MoUVjlZZE6pIwE8QMINvMsoA1BEng+xUbmdnRQCvgk7Jt7v6Dcu//GMipmBwkegvXbecvb+dx4aCOnN1Xw6CL1DWhlZjcvRi4FpgGLAQmuPsCMxtnZiPLNR0DjHd39QBqkX0lpdw0cQ4tG6dx53dUWhKpi0J9UM7dpwJTK2y7vcL6nYc4xhPAE0kOTY7Qw+8tY/6a7fz18uNp1SQ96nBEJAQaakMO25L1O/jzW3mcP6ADI/p3iDocEQmJEoQcluKSUm56bg5NMxowbqRKSyJ1mcZiksPy6AcrmJO/jb98/zgymzaMOhwRCVFCPQgz6x92IFLz5W0o5ME3lzCi31Gcf6xKSyJ1XaIlpr+a2Wdm9nMzaxlqRFIjlZQ6N02cQ+P0VO7+bn9MkyyJ1HkJJQh3PxX4AcGDbzPN7F9mdnaokUmN8viHK/hi1VbuGtmPts1UWhKpDxK+SO3uS4FfA78EzgD+bGaLzOyisIKTmmF5QSH//fpizurTnpEDO0YdjohUk0SvQQwwswcJHngbDnzH3fvElh8MMT6JWGmp88tJc2nYIIV7R6m0JFKfJHoX01+AR4Hb3H132UZ3X2tmvw4lMqkR/vHJl8z4cgu/v3Qg7ZpnRB2OiFSjQyaI2MQ/q939qXjvV7ZdInDllKQebuWmndz32iKGHd2Wi46PO1K7iNRhhywxxYbgzjQzjadQj5SWOjdPnEtaSgr3XnSsSksi9VCiJaaVwEdmNhnYWbbR3f8QSlRSJZc9HAyI++zVJx/xsZ6evpLpKzZz/8UD6NCi0REfT0Rqn0QTxNrYKwVoFl44UhOs3ryL3766iNN7t+XSnM5RhyMiEUkoQbj7XWEHIjWDu3PL83NJMeO3Ki2J1GsJJQgzawvcDPQDvr6Vxd2HhxSXROSZz1bzUd4m7hnVn04tVVoSqc8SfVDuaWARkAXcBXxJMGOc1CFrtu7m3qkLOaVnJt8f3DXqcEQkYokmiEx3/xuwz93fc/efACeFGJdUM3fnlklzKXXnvosHqLQkIglfpN4X+7nOzM4nuGBdd65eJvn5gdrouZn5fLB0I+Mu7EeX1o2jDkdEaoBEE8RvzKwF8Avgf4DmwA2hRSXVat223dz9Si5Dslpz+ZBuUYcjIjVEoncxvRJb3AYMS/TgZjYC+BOQCjzm7r+r8P6D5Y7XGGjn7i3NbBDwfwSJqAS4x92fTfRzJXHuzm3Pz6O41Ln/kgGkpKi0JCKBRO9i+jvgFbfHrkVUtk8q8BBwNpAPzDCzye6eW27/G8q1vw44Lra6C/ihuy81s47A52Y2zd23JhKvJO75WWt4Z3EBt1/Ql26ZTaIOR0RqkERLTK+UW84ARhFchziYwUCeuy8HMLPxwIVAbiXtxwB3ALj7krKNsQEBNwBtASWIJFq/fQ93vbyAE7u34sendI86HBGpYRItMU0qv25mzwBvHmK3TsDqcuv5wJB4Dc2sG8EttG/HeW8wkA4sSyRWSYy786sX5rO3uJT7Lxmo0pKIHCDhCYMqyAYOdaN8vG+cA8pUMaOBibGBAb85gFkH4CngSncvPeADzMaa2Uwzm1lQUJBA2FJm8py1vLlwPTeeczRZbVRaEpEDJTph0A4z2172Al4mmFnuYPIJpigt05nKy1KjgWcqfGZzYArwa3f/NN5O7v6Iu+e4e07btm0TORUBCnbs5Y7JCziua0t+cmpW1OGISA2VaImpKgP0zQCyzSwLWEOQBL5fsZGZHQ20Aj4pty0deAF40t2fq8JnSyXcnf96cT67ikp44JKBpKq0JCKVSLQHMSr2HETZeksz++7B9nH3YuBaYBrBVKUT3H2BmY0zs5Hlmo4Bxrt7+fLT94DTgR+b2ezYa1CC5yQHMWXeOl5b8BU3nNWbXu2aRh2OiNRgid7FdIe7v1C24u5bzewO4MWD7eTuU4GpFbbdXmH9zjj7/RP4Z4KxSYI2Fe7l9pcWMLBzC646TaUlETm4RC9Sx2uXaHKRGuL2yQso3FPM/ZcMpEFqVe9PEJH6ItFviZlm9gcz62lmPWJPQH8eZmCSXK/NX8eUuev49zN7cfRRmvNJRA4t0QRxHVAEPAtMAHYD14QVlCTXlp1F/PrF+fTr2Jyrz+gZdTgiUkskehfTTuCWkGORkNz58gK27trHUz8dQppKSyKSoETvYnrDzFqWW29lZtPCC0uS5Y3c9bw0ey3XDu9Fnw7Now5HRGqRRP+cbFN+oDx33wK0CyckSZatu4q47YV5HHNUM37+rV5RhyMitUyiCaLUzL4eWsPMulP5sBlSQ4x7JZfNO4v470sHkt5ApSUROTyJ3qr6K+BDM3svtn46MDackCQZ3l60nudnreG64b3o36nFoXcQEakg0YvUr5lZDkFSmA28RHAnk9RA23bv47bn59O7fVOuHa7SkohUTaITBv0bcD3BgHuzgZMIxk4aHl5oUlX3TMmloHAvj/zwBBo2SI06HBGppRItTF8PnAisdPdhBDO/aXztGmjrriImzMxn7Ok9GNC55aF3EBGpRKIJYo+77wEws4buvgg4OrywpCqKS50VG3fRq11Trj8zO+pwRKSWS/QidX7sOYgXgTfMbAuHnnJUqtnqzbsoKinl/ksGkJGm0pKIHJlEL1KPii3eaWbvAC2A10KLSg7bR3kb2bBjLx1aZHB811ZRhyMidcBhj8jq7u8dupVUp517i/nlpLlkpKXQuWWjqMMRkTpCT0/VAfe9tog1W3fTo00TUjRDnIgkiRJELffJsk08+clKrjwli2YZaVGHIyJ1iBJELbarKCgtdctszE3n6qYyEUmuUBOEmY0ws8VmlmdmBwwXbmYPlptzeomZbS333o/MbGns9aMw46ytHpi2mFWbd3HfxQNolK67lkQkuUKbNtTMUoGHgLOBfGCGmU1299yyNu5+Q7n21xE8gIeZtQbuAHIIBgX8PLbvlrDirW1mfLmZJz7+kh+d3I2TemRGHY6I1EFh9iAGA3nuvtzdi4DxwIUHaT8GeCa2fC7whrtvjiWFN4ARIcZaq+wuKuHmiXPp3KoRN484JupwRKSOCjNBdAJWl1vPj207gJl1A7KAtw933/roD28sZsXGndx30QCaNAytEygi9VyYCSLe/ZaVzSExGpjo7iWHs6+ZjTWzmWY2s6CgfgwN9fnKLfztwxX8YEhXTunVJupwRKQOCzNB5ANdyq13pvLhOUbzTXkp4X3d/RF3z3H3nLZt2x5huDXfnn0l3DxxDh1aNOLW8/pEHY6I1HFhJogZQLaZZZlZOkESmFyxkZkdDbQiGD68zDTgnNjc162Ac2Lb6rU/vrmUZQU7+e1Fx9JUpSURCVlo3zLuXmxm1xJ8sacCj7v7AjMbB8x097JkMQYY7+5ebt/NZnY3QZIBGOfum8OKtTaYs3orj7y/jMtyunB677rfWxKR6IX6Z6i7TwWmVth2e4X1OyvZ93Hg8dCCq0X2Fpdw43NzaN88g19doNKSiFQP1Slqgf95K4+lGwr5+5Un0lzDaYhINdFQGzXc/DXb+L/3lnHx8Z0ZdnS7qMMRkXpECaIGKyou5cbn5pDZJJ3bL+gbdTgiUgAnEjAAAA4VSURBVM+oxFSDPfROHou+2sFjP8yhRWOVlkSkeqkHUUPlrt3OQ+/k8d1BHTmrb/uowxGRekgJogbaVxKUllo2TueO7/SLOhwRqadUYqqB/vruMnLXbeevl59AqybpUYcjIvWUehA1zOKvdvDnt5dywYAOjOh/VNThiEg9pgRRgxSXlHLTxDk0z0jjrpEqLYlItFRiqkEe+WA5c/O38dD3jyezacOowxGRek49iBoib8MO/vjGUr7d/yjOH9Ah6nBERJQgaoKSUufG5+bSpGEq4y7sH3U4IiKASkw1wt8+XM7s1Vv50+hBtG2m0pKI1AzqQURseUEhv399CWf3bc/IgR2jDkdE5GtKEBEqKXVunjiXjLRU7vluf8zizbQqIhINlZgi9MTHXzJz5Rb+8L2BtGueccTHe/bqk5MQlYhIQD2IiHy5cScPTFvE8GPaMeq4TlGHIyJyACWICJSWOjdPmktaagr3jjpWpSURqZGUICLwz+kr+WzFZv7rgr4c1eLIS0siImEINUGY2QgzW2xmeWZ2SyVtvmdmuWa2wMz+VW77/bFtC83sz1ZH/sxevXkXv3t1Eaf3bsulJ3SOOhwRkUqFdpHazFKBh4CzgXxghplNdvfccm2ygVuBoe6+xczaxbafAgwFBsSafgicAbwbVrzVwd355aS5pJjxu4tUWhKRmi3MHsRgIM/dl7t7ETAeuLBCm6uAh9x9C4C7b4htdyADSAcaAmnA+hBjrRb/+mwVHy/bxG3n9aFjy0ZRhyMiclBhJohOwOpy6/mxbeX1Bnqb2Udm9qmZjQBw90+Ad4B1sdc0d19Y8QPMbKyZzTSzmQUFBaGcRLLkb9nFvVMWcmqvNowZ3CXqcEREDinMBBGvfuIV1hsA2cC3gDHAY2bW0sx6AX2AzgRJZbiZnX7Awdwfcfccd89p27ZtUoNPJnfn1ufn4cBvVVoSkVoizASRD5T/U7kzsDZOm5fcfZ+7rwAWEySMUcCn7l7o7oXAq8BJIcYaqgkzV/PB0o3c+u1j6NK6cdThiIgkJMwEMQPINrMsM0sHRgOTK7R5ERgGYGZtCEpOy4FVwBlm1sDM0gguUB9QYqoN1m3bzW9eWchJPVrzgyHdog5HRCRhoSUIdy8GrgWmEXy5T3D3BWY2zsxGxppNAzaZWS7BNYeb3H0TMBFYBswD5gBz3P3lsGINS1lpqbjUuf/igaSkqLQkIrVHqGMxuftUYGqFbbeXW3bgP2Ov8m1KgKvDjK06TJq1hncXF3DHd/rSNVOlJRGpXfQkdUjWb9/DuJcXMLh7a350cveowxEROWxKECFwd371wjz2Fpdy3yUDVFoSkVpJCSIEL81ey5sLN3DTuUeT1aZJ1OGIiFSJEkSSbdixhztfXsDxXVty5dCsqMMREakyJYgkcnf+68X57Coq4f5LBpKq0pKI1GJKEEn0ytx1TFuwnv88uze92jWNOhwRkSOiBJEkGwv3csfkBQzs3IJ/O1WlJRGp/ZQgkuSOlxZQuKeYBy4dSINU/VpFpPbTN1kSvDpvHVPmreP6s7Lp3b5Z1OGIiCSFEsQR2ryziP96aT79OzVn7Ok9og5HRCRpQh1qoz64c/ICtu3ex1M/HUKaSksiUofoG+0IvL7gKybPWcu1w7Lp06F51OGIiCSVEkQVbd1VxK9enE+fDs35+bCeUYcjIpJ0KjFV0bhXctmys4gnrjxRpSURqZP0zVYFby9az/Oz1vDzb/WkX8cWUYcjIhIKJYjDtG33Pm59fh5Ht2/GtcOzow5HRCQ0KjEdpnum5LKxsIhHf5hDegPlVxGpu/QNdxjeXbyBCTPzufr0Hgzo3DLqcEREQhVqgjCzEWa22MzyzOyWStp8z8xyzWyBmf2r3PauZva6mS2Mvd89zFgPZceeoLTUq11T/v1MlZZEpO4LrcRkZqnAQ8DZQD4ww8wmu3tuuTbZwK3AUHffYmbtyh3iSeAed3/DzJoCpWHFmoh7py5i/fY9TPp/p5CRlhplKCIi1SLMHsRgIM/dl7t7ETAeuLBCm6uAh9x9C4C7bwAws75AA3d/I7a90N13hRjrQX24dCPPfLaKq07rwXFdW0UVhohItQozQXQCVpdbz49tK6830NvMPjKzT81sRLntW83seTP7wsweiPVIql3h3mJ+OWkuPdo04Yaze0cRgohIJMJMEPGmU/MK6w2AbOBbwBjgMTNrGdt+GnAjcCLQA/jxAR9gNtbMZprZzIKCguRFXs59ry5i7bbdPHDpAJWWRKReCTNB5ANdyq13BtbGafOSu+9z9xXAYoKEkQ98EStPFQMvAsdX/AB3f8Tdc9w9p23btkk/gY+XbeSpT1fyk6FZnNCtddKPLyJSk4WZIGYA2WaWZWbpwGhgcoU2LwLDAMysDUFpaXls31ZmVvatPxzIpRrtKirmlknz6J7ZmBvPObo6P1pEpEYILUHE/vK/FpgGLAQmuPsCMxtnZiNjzaYBm8wsF3gHuMndN7l7CUF56S0zm0dQrno0rFjjuf+1xazavIv7Lh5Ao3SVlkSk/jH3ipcFaqecnByfOXNmUo712YrNXPbIJ/zo5O7cObJfUo4pIlITmdnn7p4T7z09SV3B7qISbp44h86tGnHzCJWWRKT+0lhMFfz+9cV8uWkX/7pqCI3T9esRkfpLPYhyPl+5mb99tIIfDOnKKT3bRB2OiEiklCBi9uwr4aaJc+nYohG3ntcn6nBERCKnGgpw2cOfsGrzLtZt28NTPx1M04b6tYiIqAdBMJzGum17GH1iF07LTv4DdyIitVG9TxB7i0tYXrCT9NQUbjtfpSURkTL1PkFs2L4XgKw2jWmekRZxNCIiNUe9L7Z3ad2YYzs1xyze2IIiIvVXve9BAEoOIiJx1PseBMCzV58cdQgiIjWOehAiIhKXEoSIiMSlBCEiInEpQYiISFxKECIiEpcShIiIxKUEISIicSlBiIhIXEoQIiISl7l71DEkhZkVACuP4BBtgI1JCidKdeU8QOdSU9WVc6kr5wFHdi7d3D3uPAd1JkEcKTOb6e45UcdxpOrKeYDOpaaqK+dSV84DwjsXlZhERCQuJQgREYlLCeIbj0QdQJLUlfMAnUtNVVfOpa6cB4R0LroGISIicakHISIicSlBxJjZ3WY218xmm9nrZtYx6piqysweMLNFsfN5wcxaRh1TVZnZpWa2wMxKzazW3XFiZiPMbLGZ5ZnZLVHHcyTM7HEz22Bm86OO5UiYWRcze8fMFsb+bV0fdUxVZWYZZvaZmc2JnctdST2+SkwBM2vu7ttjy/8O9HX3n0UcVpWY2TnA2+5ebGb3Abj7LyMOq0rMrA9QCjwM3OjuMyMOKWFmlgosAc4G8oEZwBh3z400sCoys9OBQuBJd+8fdTxVZWYdgA7uPsvMmgGfA9+tjf9dLJgvuYm7F5pZGvAhcL27f5qM46sHEVOWHGKaALU2c7r76+5eHFv9FOgcZTxHwt0XuvviqOOoosFAnrsvd/ciYDxwYcQxVZm7vw9sjjqOI+Xu69x9Vmx5B7AQ6BRtVFXjgcLYalrslbTvLiWIcszsHjNbDfwAuD3qeJLkJ8CrUQdRT3UCVpdbz6eWfhHVVWbWHTgOmB5tJFVnZqlmNhvYALzh7kk7l3qVIMzsTTObH+d1IYC7/8rduwBPA9dGG+3BHepcYm1+BRQTnE+Nlci51FIWZ1ut7ZnWNWbWFJgE/EeFCkKt4u4l7j6IoFIw2MySVv5rkKwD1QbuflaCTf8FTAHuCDGcI3KoczGzHwEXAGd6Db/QdBj/XWqbfKBLufXOwNqIYpFyYvX6ScDT7v581PEkg7tvNbN3gRFAUm4kqFc9iIMxs+xyqyOBRVHFcqTMbATwS2Cku++KOp56bAaQbWZZZpYOjAYmRxxTvRe7sPs3YKG7/yHqeI6EmbUtu0vRzBoBZ5HE7y7dxRRjZpOAownumFkJ/Mzd10QbVdWYWR7QENgU2/RpLb4jaxTwP0BbYCsw293PjTaqxJnZecAfgVTgcXe/J+KQqszMngG+RTBy6HrgDnf/W6RBVYGZnQp8AMwj+P8d4DZ3nxpdVFVjZgOAfxD8+0oBJrj7uKQdXwlCRETiUYlJRETiUoIQEZG4lCBERCQuJQgREYlLCUJEROJSghA5DGZWeOhWB91/opn1iC03NbOHzWxZbCTO981siJmlx5br1YOsUvMoQYhUEzPrB6S6+/LYpscIBr/Ldvd+wI+BNrGB/d4CLoskUJEYJQiRKrDAA7Exo+aZ2WWx7Slm9r+xHsErZjbVzC6J7fYD4KVYu57AEODX7l4KEBv1dUqs7Yux9iKRURdWpGouAgYBAwmeLJ5hZu8DQ4HuwLFAO4KhpB+P7TMUeCa23I/gqfCSSo4/HzgxlMhFEqQehEjVnAo8ExtJcz3wHsEX+qnAc+5e6u5fAe+U26cDUJDIwWOJoyg2oY1IJJQgRKom3lDeB9sOsBvIiC0vAAaa2cH+H2wI7KlCbCJJoQQhUjXvA5fFJmtpC5wOfEYw5ePFsWsR7QkGtyuzEOgF4O7LgJnAXbHRRTGz7LI5MMwsEyhw933VdUIiFSlBiFTNC8BcYA7wNnBzrKQ0iWAeiPkE82hPB7bF9pnC/gnj34CjgDwzmwc8yjfzRQwDat3oolK3aDRXkSQzs6axSeQzCXoVQ939q9h4/e/E1iu7OF12jOeBW2vxfNxSB+guJpHkeyU2iUs6cHesZ4G77zazOwjmpV5V2c6xyYVeVHKQqKkHISIicekahIiIxKUEISIicSlBiIhIXEoQIiISlxKEiIjEpQQhIiJx/X/Smsz2nh4aZwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "#train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "#train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "#train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "#train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    #plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuary' )\n",
    "plt.savefig('LogisticGridSearchCV_C_1.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l2\n",
      "0.1\n"
     ]
    }
   ],
   "source": [
    "print(grid.best_params_['penalty'])\n",
    "print(grid.best_params_['C'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROC曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve,auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_roc(lables,predict_prob):\n",
    "    false_positive_rate,true_positive_rate,thresholds=roc_curve(lables,predict_prob)\n",
    "    roc_auc=auc(false_positive_rate,true_positive_rate)\n",
    "    \n",
    "    plt.plot(false_positive_rate,true_positive_rate,'b',label='AUC=%0.4f'%roc_auc)\n",
    "    plt.legend(loc='lower right',fontsize=16)\n",
    "    plt.plot([0,1],[0,1],'r--')\n",
    "    plt.ylabel('TPR',fontsize=16)\n",
    "    plt.xlabel('FPR',fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(6,6))\n",
    "plot_roc(y_train,test_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.65104167, 0.74739583],\n",
       "       [0.70703125, 0.76171875],\n",
       "       [0.76302083, 0.77473958],\n",
       "       [0.77213542, 0.77083333],\n",
       "       [0.77083333, 0.77083333],\n",
       "       [0.77083333, 0.77083333],\n",
       "       [0.77083333, 0.77083333]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(test_means).reshape(n_Cs,number_penaltys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.65104167, 0.74739583, 0.70703125, 0.76171875, 0.76302083,\n",
       "       0.77473958, 0.77213542, 0.77083333, 0.77083333, 0.77083333,\n",
       "       0.77083333, 0.77083333, 0.77083333, 0.77083333])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
